import os
import random
import re
import time
import traceback

import pymysql
import requests_html
import requests

if __name__ == '__main__':
    conn = pymysql.Connect(host = '10.170.128.56',
                           port = 3306,
                           user = 'root',
                           passwd = '123456',
                           db = 'books',
                           charset='utf8mb4')
    cur = conn.cursor()
    user_agent=requests_html.user_agent()
    r=open("C:\\Users\\diao\\Desktop\\all_ids.txt","r",encoding="utf-8")
    sql1="select book_id from books_image where id=(select max(id) from books_image)"
    cur.execute(sql1)
    cur_book_id=cur.fetchone()
    if cur_book_id:
        cur_book_id=cur_book_id[0]
        flag=False
        #未爬取id
        os.remove("C:\\Users\\diao\\Desktop\\tmp_ids.txt")
        ids=open("C:\\Users\\diao\\Desktop\\tmp_ids.txt","a",encoding='utf-8')
        #id汇总文件
        for id in r:
            #print(id.replace('\n', ''))
            if flag == True:
                ids.write(id)
            else:
                if id.replace('\n', '') !=cur_book_id:
                    continue
                else:
                    flag=True
        ids.close()
        r.close()
    ids=open("C:\\Users\\diao\\Desktop\\tmp_ids.txt","r",encoding='utf-8')
    for line in ids:
        try:
            book_id=line.replace('\n','')
            # print(book_id)
            url="https://www.allhistory.com/book/detail?id="+book_id
            session=requests_html.HTMLSession()
            headers={"User-Agent":user_agent,
                "Cookie":"sid=c33acb969ed91b06e2a98e55d405c4e8; sys_lang=zh; UM_distinctid=172f369309e477-0121f77c4e3b6-551d3911-144000-172f369309f4b2; _cbshow=true; CNZZDATA1277615859=1385762309-1593221660-%7C1593248835; aha=WEBmsg8618634976469LOGIN; bearer=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJBSDEzNDI1MjY4MDczNTgiLCJjcmVhdGVkIjoxNTkzMjQ5ODQ2ODEzLCJleHAiOjE1OTU4NDE4NDYsInVzZXJpZCI6MTM0MjUyNn0.BTNaUgUmhnMuPuSNBemw3m5vJsYfnHiGluQUnpnJ51-TcFUip1EbKigeb2nIZge4AJSQwzwy1GSyR5iChoV98A; free=free; page.prev.id=11v96nj13kbxfluzc; page.id=11v96nj1skbxftmmm; refer.url=https%3A%2F%2Fwww.allhistory.com%2Fbook%2Fhome-list%2F; connect.sid=s%3A3HhrLMtGhtlwHp0OTICuZTiO1UqGwWS9.nJ%2B%2B1tpfya8vb9Yntezcy77jE5YEJk6HAg6bRZkinSE"
             }
            r=session.get(url,headers=headers)
            r.html.render(sleep=1,timeout=15)
            data=r.html.html
            pat=r'<div class="book-cover bookimg" style="background-image: url\(//(.*?)\)"></div>'
            image_url=re.findall(pat,data)
            if image_url:
                print("有image book_id: "+book_id)
                image_url="http://"+image_url[0]
                image=requests.get(image_url)
                w=open("C:\\Users\\diao\\Desktop\\ancient_books\\images\\"+book_id+".jpeg","wb")
                w.write(image.content)
                w.close()
                sql="insert into books_image(book_id,image) values(%s,%s)"
                values=(book_id,image.content)
                cur.execute(sql,values)
                conn.commit()
                del image
            else:
                print("无image book_id: "+book_id)
                sql="insert into books_image(book_id) values(%s)"
                values=(book_id)
                cur.execute(sql,values)
                conn.commit()
                no_image=open("C:\\Users\\diao\\Desktop\\ancient_books\\noimage_ids.txt","a",encoding="utf-8")
                no_image.write(book_id+'\n')
                no_image.close()
            del data
            time.sleep(random.randint(2,4))
        except:
            traceback.print_exc()
            err=open("C:\\Users\\diao\\Desktop\\err_image_ids.txt","a",encoding="utf-8")
            err.write(book_id+'\n')
            err.close()
            continue

    # sql="select book_id,image from books_image limit 1"
    # cur.execute(sql)
    # data=cur.fetchone()
    # book_id=data[0]
    # image=data[1]
    # w=open("C:\\Users\\diao\\Desktop\\ancient_books\\images\\"+book_id+".jpeg","wb")
    # w.write(image)
    # w.close()
    cur.close()
    conn.close()
    ids.close()



